{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic 回归——Pima Indians Diabetes Data Set\n",
    "数据说明： Pima Indians Diabetes Data Set（皮马印第安人糖尿病数据集） 根据现有的医疗信息预测5年内皮马印第安人糖尿病发作的概率。\n",
    "\n",
    "数据集共9个字段: \n",
    "    0列为怀孕次数； \n",
    "    1列为口服葡萄糖耐量试验中2小时后的血浆葡萄糖浓度； \n",
    "    2列为舒张压（单位:mm Hg） \n",
    "        3列为三头肌皮褶厚度（单位：mm） \n",
    "        4列为餐后血清胰岛素（单位:mm） \n",
    "        5列为体重指数（体重（公斤）/ 身高（米）^2） \n",
    "        6列为糖尿病家系作用 \n",
    "        7列为年龄 \n",
    "        8列为分类变量（0或1）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#评价指标为logloss\n",
    "from sklearn.metrics import log_loss  \n",
    "\n",
    "from matplotlib import pyplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.866045       -0.031990   \n",
       "1  -0.844885                     -1.205066       -0.528319   \n",
       "2   1.233880                      2.016662       -0.693761   \n",
       "3  -0.844885                     -1.073567       -0.528319   \n",
       "4  -1.141852                      0.504422       -2.679076   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.670643      -0.181541  0.166619   \n",
       "1                    -0.012301      -0.181541 -0.852200   \n",
       "2                    -0.012301      -0.181541 -1.332500   \n",
       "3                    -0.695245      -0.540642 -0.633881   \n",
       "4                     0.670643       0.316566  1.549303   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "train = pd.read_csv(\"./FE_pima-indians-diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "pregnants                       768 non-null float64\n",
      "Plasma_glucose_concentration    768 non-null float64\n",
      "blood_pressure                  768 non-null float64\n",
      "Triceps_skin_fold_thickness     768 non-null float64\n",
      "serum_insulin                   768 non-null float64\n",
      "BMI                             768 non-null float64\n",
      "Diabetes_pedigree_function      768 non-null float64\n",
      "Age                             768 non-null float64\n",
      "Target                          768 non-null int64\n",
      "dtypes: float64(8), int64(1)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  get labels\n",
    "y_train = train['Target']   \n",
    "X_train = train.drop([\"Target\"], axis=1)\n",
    "\n",
    "#用于特征重要性可视化\n",
    "feat_names = X_train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr= LogisticRegression(solver='liblinear')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.48797856 0.53011593 0.4562292  0.422546   0.48392885]\n",
      "cv logloss is: 0.47615970944432684\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print('logloss of each fold is: ',-loss) \n",
    "print('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 正则化的 Logistic Regression及参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='warn',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='liblinear',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
       "                         'penalty': ['l1', 'l2']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.47602910949275207\n",
      "{'C': 1, 'penalty': 'l1'}\n",
      "{'mean_fit_time': array([0.00208502, 0.00198865, 0.00159874, 0.00198898, 0.00040684,\n",
      "       0.00313239, 0.        , 0.00312471, 0.00312595, 0.        ,\n",
      "       0.00312424, 0.00312586, 0.00355582, 0.00020833]), 'std_fit_time': array([6.57610936e-04, 1.14470716e-05, 4.91196750e-04, 1.60309504e-05,\n",
      "       8.13674927e-04, 6.26478195e-03, 0.00000000e+00, 6.24942780e-03,\n",
      "       6.25190735e-03, 0.00000000e+00, 6.24847412e-03, 6.25171661e-03,\n",
      "       6.11205388e-03, 4.16660309e-04]), 'mean_score_time': array([0.00180602, 0.00119681, 0.001194  , 0.00120072, 0.        ,\n",
      "       0.        , 0.00312295, 0.        , 0.        , 0.00312257,\n",
      "       0.        , 0.        , 0.00041804, 0.        ]), 'std_score_time': array([0.00074858, 0.00039878, 0.00040127, 0.00041544, 0.        ,\n",
      "       0.        , 0.0062459 , 0.        , 0.        , 0.00624514,\n",
      "       0.        , 0.        , 0.000512  , 0.        ]), 'param_C': masked_array(data=[0.001, 0.001, 0.01, 0.01, 0.1, 0.1, 1, 1, 10, 10, 100,\n",
      "                   100, 1000, 1000],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_penalty': masked_array(data=['l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1',\n",
      "                   'l2', 'l1', 'l2', 'l1', 'l2'],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 0.001, 'penalty': 'l1'}, {'C': 0.001, 'penalty': 'l2'}, {'C': 0.01, 'penalty': 'l1'}, {'C': 0.01, 'penalty': 'l2'}, {'C': 0.1, 'penalty': 'l1'}, {'C': 0.1, 'penalty': 'l2'}, {'C': 1, 'penalty': 'l1'}, {'C': 1, 'penalty': 'l2'}, {'C': 10, 'penalty': 'l1'}, {'C': 10, 'penalty': 'l2'}, {'C': 100, 'penalty': 'l1'}, {'C': 100, 'penalty': 'l2'}, {'C': 1000, 'penalty': 'l1'}, {'C': 1000, 'penalty': 'l2'}], 'split0_test_score': array([-0.69314718, -0.62523328, -0.6359066 , -0.51207677, -0.48922476,\n",
      "       -0.48350008, -0.4881555 , -0.48797856, -0.48903192, -0.48902949,\n",
      "       -0.48913706, -0.4891444 , -0.48914575, -0.489156  ]), 'split1_test_score': array([-0.69314718, -0.6338686 , -0.64460736, -0.54322757, -0.52488387,\n",
      "       -0.52514943, -0.52917845, -0.53011593, -0.53103501, -0.53113632,\n",
      "       -0.53123069, -0.53124692, -0.53125776, -0.53125807]), 'split2_test_score': array([-0.69314718, -0.62566121, -0.63328109, -0.50673804, -0.45824627,\n",
      "       -0.46048749, -0.45561722, -0.4562292 , -0.45588924, -0.45595951,\n",
      "       -0.45592582, -0.45593526, -0.45592938, -0.45593286]), 'split3_test_score': array([-0.69314718, -0.62087342, -0.63024185, -0.48731256, -0.43335964,\n",
      "       -0.42929827, -0.42237271, -0.422546  , -0.42195413, -0.42197953,\n",
      "       -0.42192171, -0.42192491, -0.42191874, -0.42191947]), 'split4_test_score': array([-0.69314718, -0.63268716, -0.63580105, -0.52538884, -0.48686691,\n",
      "       -0.4853734 , -0.48452844, -0.48392885, -0.48411117, -0.48408647,\n",
      "       -0.48411082, -0.48410727, -0.48410677, -0.48410941]), 'mean_test_score': array([-0.69314718, -0.62766704, -0.63597526, -0.51497115, -0.47856421,\n",
      "       -0.47681232, -0.47602911, -0.4762194 , -0.47646516, -0.47649922,\n",
      "       -0.47652629, -0.47653284, -0.47653277, -0.47653626]), 'std_test_score': array([1.11022302e-16, 4.89097299e-03, 4.79089060e-03, 1.86921266e-02,\n",
      "       3.09291814e-02, 3.15479636e-02, 3.56008155e-02, 3.57224624e-02,\n",
      "       3.62946466e-02, 3.63084245e-02, 3.63664912e-02, 3.63697206e-02,\n",
      "       3.63755758e-02, 3.63758791e-02]), 'rank_test_score': array([14, 12, 13, 11, 10,  9,  1,  2,  3,  4,  5,  7,  6,  8])}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU5dn/8c81WQhL2MMaCIiggmwSIbivEKtViwtSpGqf1mq1i22tWlutuLZ9+tRfK1rUulRRpG5FVBBxV7aAgCyiiCJhl33Pdv3+mEFjGMhkGU5m8n2/XueVOefcZ+Z7iM6Vs923uTsiIiIVhYIOICIidZMKhIiIRKUCISIiUalAiIhIVCoQIiISVWrQAWpL69atvUuXLkHHEBFJKHPmzPnK3bOirUuaAtGlSxcKCgqCjiEiklDMbMWB1sX1FJOZ5ZvZUjNbZmY3Rln/NzObF5k+MbMt5dZdZmafRqbL4plTRET2F7cjCDNLAcYAZwKFwGwzm+jui/e1cffryrX/GdA/8rolcCuQCzgwJ7Lt5njlFRGRb4vnEcRAYJm7L3f3ImA8cN5B2o8Ano68HgpMdfdNkaIwFciPY1YREakgntcgOgIry80XAoOiNTSzHKAr8MZBtu0YZbsrgSsBOnfuXPPEIpI0iouLKSwsZM+ePUFHqRMyMjLIzs4mLS0t5m3iWSAsyrIDdfx0CfCsu5dWZVt3fxB4ECA3N1edSonI1woLC8nMzKRLly6YRftKqT/cnY0bN1JYWEjXrl1j3i6ep5gKgU7l5rOB1QdoewnfnF6q6rYiIvvZs2cPrVq1qvfFAcDMaNWqVZWPpuJZIGYD3c2sq5mlEy4CEys2MrMjgBbA9HKLpwBDzKyFmbUAhkSWiYjETMXhG9X5t4hbgXD3EuBawl/sS4AJ7r7IzEab2bnlmo4Axnu5fsfdfRNwO+EiMxsYHVlW60rLnLteWcLKTbvi8fYikkCGj53O8LHTK29YT8T1OQh3f8Xde7h7N3e/M7LsFnefWK7NH919v2ck3P0Rdz88Mj0ar4wrNu5k/KwvGfbAByxctTVeHyMi9VCTJk2+fp2fn0/z5s0555xzora95ppr6NevHz179qRhw4b069ePfv368eyzz1bpM+fOncvkyZNrlHufet8X02FZTXj26uNICxkXj53Om0vXBx1JRJLQ9ddfzxNPPHHA9WPGjGHevHm88sordOvWjXnz5jFv3jwuvPDCKn2OCkQt69E2kxeuOZ4urRrzo8cLeGb2l0FHEpEkc/rpp5OZmVmtbT/99FOGDh3KgAEDOOmkk/jkk08AGD9+PEcffTR9+/bl1FNPZffu3YwePZpx48ZV6+ijoqTpi6mm2jbNYMJVg7n6yTnc8NxHrNqyh+vO6K6LXCJJ4LaXFrF49bZK2y1eE24Ty3WInh2acut3e9U4WyyuvPJKHn74Ybp168b777/Ptddey2uvvcZtt93GW2+9Rdu2bdmyZQsNGzbklltuYeHChdx77701/lwViHKaNEjlkcuP5XfPf8Tfp33K6i27uXtYb9JSdKAlIsHYsmULM2bM4IILLvh6WUlJCQDHH388P/jBD7jooosYNmxYrX+2CkQFaSkh/nxhHzo0b8j/m/Yp67bt4f6Rx5CZEfvThyJSt8T6l/6+I4dnfjI4nnGqxN1p3bo18+bN22/dQw89xMyZM5k0aRJ9+/ZlwYIFtfrZ+tM4CjPjujN78KcLevPBZxu5eOwM1m3T4/oicui1aNGC9u3b88ILLwBQVlbG/PnzAVi+fDl5eXncfvvttGjRglWrVpGZmcn27dtr5bNVIA5i+LGd+ddluazYuJPvjXmfT9bVzj+6iNQ/J554IhdddBHTpk0jOzubKVNif/Z3/Pjx/POf/6Rv37706tWLSZMmAXDdddfRu3dvevfuzRlnnMHRRx/Naaedxvz58+nfv3+NL1JbuefTElpubq7Ha8Cghau2csVjs9lTXMqDo3IZ3K1VXD5HRGrPkiVLOOqoo6q0TV08xVSbov2bmNkcd8+N1l5HEDE4umMznr/6ONo2zeCyR2Yxcb66hRJJRs/8ZHDSFofqUIGIUaeWjXjuquPo17k5P3/6Q8a+/RnJcvQlIhKNCkQVNGuUxr9/OJCz+7Tn7lc/5taJiygtU5EQkeSk21yrKCMthX9c0p8OzTJ46N3PWbN1D3+/pD8N01OCjiYiUqt0BFENoZBx89k9+eN3e/L6knWMeGgGG3fsDTqWiEitUoGogcuP78oDIwewZM02LnjgA774amfQkUSkJh49OzwJoAJRY/lHt+OpHw9i6+5ihj3wAR9+uTnoSCJSR+zr7nvevHkMHjyYXr160adPH5555pn92tbF7r51DaIWDMhpyXNXH8flj85mxEMz+Psl/RnSq13QsUSkjmjUqBH//ve/6d69O6tXr2bAgAEMHTqU5s2bf91mzJgxAHzxxRecc845UbvWiMXcuXNZuHAh+fn5Nc6tI4haclhWE57/6XEc0TaTq56cwxPTvwg6kojUET169KB79+4AdOjQgTZt2rBhw4aYt1d330mgdZMGPH1lHj976kP+8N9FrNqyh98OPYJQSF2GiwTq1Rth7UeVt1sb6ewulusQ7XrDWfdUOcqsWbMoKiqiW7duMW+j7r6TRKP0VMaOGsCtExfxz7c/Y/WW3fzloj40SNVtsCL13Zo1axg1ahSPP/44oVBsJ3DU3XeSSU0Jccf5R9OxRUP+PHkp67fvYeyoXJo1VJfhIoGI9S/9fUcOV7xc6xG2bdvG2WefzR133EFeXl7M26m77yRkZvz0lMP52/C+zFmxmYv++QGrtuwOOpaIBKCoqIjvfe97X/+1XxXq7juJfa9/No9fMZA1W/Yw7P73WbR6a9CRROQQmzBhAu+88w6PPfbY17evVuUuJXX3XUPx7O67Nny8dhtXPDqb7XtKeODSYzixe1bQkUSSWnW6+47nKaa6QN1911FHtmvK8z89juwWDbni0dk8O6cw6EgiUtEVLydtcagOFYhDqH2zhky4ajCDDmvJb/4zn79P+1RdhotInaUCcYg1zUjj0csHMqx/R/5v6ifc9PxHFJeWBR1LJCnpD7BvVOffQre5BiA9NcRfL+5Lh+YNue/NZazdtocx3z+Gxg306xCpLRkZGWzcuJFWrVphVr8fVnV3Nm7cSEZGRpW20zdSQMyM3ww9gg7NG/L7Fz9i+IPTeeTyY2mTWbVfoIhEl52dTWFhYZW6tEhmGRkZZGdnV2kbFYiAfX9QZ9o1a8A14z5k2P0f8NgVAzm8TZOgY4kkvLS0NLp27Rp0jISmaxB1wGlHtuWZn+Sxp7iUCx74gNlfbAo6kohIfAuEmeWb2VIzW2ZmNx6gzcVmttjMFpnZU+WWl5rZvMg0MZ4564I+2c15/urjadU4nZEPz+TlBWuCjiQi9VzcCoSZpQBjgLOAnsAIM+tZoU134CbgeHfvBfyy3Ord7t4vMp0br5x1SedWjXju6uPo3bEZ1z49l4ffXR50JBGpx+J5BDEQWObuy929CBgPnFehzY+BMe6+GcDd18cxT0Jo0TidcT8aRH6vdtzx8hJue2kRpWW6VU9EDr14FoiOwMpy84WRZeX1AHqY2ftmNsPMyg+BlGFmBZHl50f7ADO7MtKmIJnuVMhIS+G+7x/DD4/vyqPvf8G1T81lT3Fp0LFEpJ6JZ4GIduNxxT+FU4HuwCnACOBhM9s3Bl/nSP8g3wfuNbP9Rtdw9wfdPdfdc7Oykqtvo5SQcct3e/L7s49i8qK1jHx4Jpt3FgUdS0TqkXgWiEKgU7n5bGB1lDb/dfdid/8cWEq4YODuqyM/lwNvAf3jmLXO+tGJh3HfiGP4aNVWLnjgA1Zu2hV0JBGpJ+JZIGYD3c2sq5mlA5cAFe9GehE4FcDMWhM+5bTczFqYWYNyy48HFscxa512dp/2jPvRIDbuLOJ797/PgsItQUcSkXogbgXC3UuAa4EpwBJggrsvMrPRZrbvrqQpwEYzWwy8CVzv7huBo4ACM5sfWX6Pu9fbAgFwbJeWPHf1cWSkpTB87Aze+Hhd0JFEJMlpPIgEs377Hn742GwWr97GHef35vuDOgcdSUQSmMaDSCJtMjN45srBnNQji9+98BH/O2WpeqwUkbhQgUhAjRuk8vAPcrnk2E7c9+Yyfj1hPkUlZQwfO53hY6cHHU9EkoQ660tQqSkh7h7Wm47NG/LXqZ+wbvseSsrKSA2p5otI7VCBSGBmxs9O70775g258bkFpKWEOKJdZtCxRCRJ6M/NJHDhgGweveJY9paUsnj1Nrbs0gN1IlJzKhBJ4sTuWRzZLpOi0jL+U1AYdBwRSQIqEEkkMyONJg1SGTdzBWXq4E9EakgFIsm0bdqALzbu4r1lXwUdRUQSnAoEwKNnh6ck0LJxOq0ap/PkjBVBRxGRBKe7mJLIMz8ZDMCfJn/M2Lc/Y/WW3XRo3jDgVCKSqHQEUVYGO9ZDaXHQSWrN9wd2xoGnZ30ZdBQRSWAqEJs/h42fwJbkOSXTqWUjTj2iDeNnr6SopCzoOCKSoFQgWnWDzPawYy2sXRh0mlozKi+HDdv38tritUFHEZEEpQIB0KwzhFJh8o2QJB3fndQji+wWDXWxWkSqTQUCICUNmneGL96FjycFnaZWpISMkYNymLF8E5+u2x50HBFJQCoQAFe8DNfMhqyj4LXfQ/GeoBPViotzs0lPCekoQkSqRQVin5RUyL8bNn8BM+4POk2taNWkAd/p3Y7n565i596SoOOISIJRgSiv26lwxNnw7l9he3Jc3B01OIfte0v477zVQUcRkQSjAlHRkNuhZC9MGx10klpxTOcWHNkukydnrNDIcyJSJSoQFbXqBoN/CvPGwao5QaepMTNj1OAcFq/ZxtwvtwQdR0QSiApENCf+Bhq3gck3JcVtr+f36xju5VUXq0WkClQgosloCqffAitnwsLngk5TY40bpDLsmI5MWrCGTTs1mJCIxEYF4kD6jYT2fWHqLVC0M+g0NXZpXg5FpWVMKFgZdBQRSRAqEAcSCkH+n2DbKnj/70GnqbEebTMZ2LWlBhMSkZipQBxMzmDoNQzevxe2JP5f3qPycli5aTdvf7oh6CgikgBUICpzZuR219dvDTZHLRjaqx2tmzTQxWoRiYkKRGWad4LjfxG+WL1ietBpaiQ9NcQlx3Zi2sfrWblpV9BxRKSOU4GIxfG/gKYdYfIN4QGGEtiIQZ0xNJiQiFROBSIW6Y3hjNtgzfzwA3QJrGPzhpx2ZFsmFKxkb0lp0HFEpA5TgYhV7wshe2C4C44924JOUyOjBufw1Y4iJi9Mjv6mRCQ+4logzCzfzJaa2TIzu/EAbS42s8VmtsjMniq3/DIz+zQyXRbPnDExg7PugZ3r4d3/DTpNjZx4eGtyWjVi3AydZhKRA4tbgTCzFGAMcBbQExhhZj0rtOkO3AQc7+69gF9GlrcEbgUGAQOBW82sRbyyxqzjAOj7fZjxAGxaHnSaaguFjJGDOjPri018vDaxj4ZEJH7ieQQxEFjm7svdvQgYD5xXoc2PgTHuvhnA3ddHlg8Fprr7psi6qUB+HLPG7oxbISUdXvtD0Elq5KIBnUhPDekoQkQOKJ4FoiNQ/umywsiy8noAPczsfTObYWb5VdgWM7vSzArMrGDDhkP08FdmOzjxV+GhSZe/dWg+Mw5aNE7nnD7teX5uITs0mJCIRBHPAmFRllXs4yEV6A6cAowAHjaz5jFui7s/6O657p6blZVVw7hVkHcNNM8J9/ZamrhfrqPycthZVMoLH64KOoqI1EHxLBCFQKdy89lAxWHNCoH/unuxu38OLCVcMGLZNjhpGTDkDli/GOY8GnSaauvXqTm9OjRlnAYTEpEo4lkgZgPdzayrmaUDlwATK7R5ETgVwMxaEz7ltByYAgwxsxaRi9NDIsvqjqO+C11OhDfvgl2bgk5TLWbGqLwcPl67nYIVm4OOIyJ1TNwKhLuXANcS/mJfAkxw90VmNtrMzo00mwJsNLPFwJvA9e6+0d03AbcTLjKzgdGRZXWHGeTfA3u2wNt/CjpNtZ3brwOZGak8qf6ZRKQCS5ZTC7m5uV5QUHDoP3jSdTDncbj6A2hz5KH//Frwx4mLGDdzBdNvOp3WTRoEHUdEDiEzm+PuudHW6Unqmjr1ZkhvAlN+l7DDk16al0NxqfPM7MTv0lxEao8KRE01bg2n3AifTYNPXws6TbUc3qYJgw9rxVMzv6RUgwmJSIQKRG0Y+GNo1T1822tJYo75PGpwDqu27Oatpesrbywi9YIKRG1ISYP8u2HTZzDrwaDTVMuZPdvSJrOBLlaLyNdUIGpL9zPh8DPDdzTtSLwhPdNSQlwysDNvfbJBgwmJCFCNAmFmITNrGo8wCW/oXVC8C968I+gk1TJiYCdCZoybqf6ZRCTGAmFmT5lZUzNrDCwGlprZ9fGNloCyesDAK8O3va5ZEHSaKmvfrCFnHNWGCQUr2VOswYRE6rtYjyB6uvs24HzgFaAzMCpuqRLZyb+FRi3DF6wT8LbXUXld2LSziFcXrgk6iogELNYCkWZmaYQLxH/dvZgonecJ0LBF+NmIFe/B4v8GnabKjuvWiq6tG/OkugEXqfdiLRBjgS+AxsA7ZpYDaKSZAxlwObQ9Gqb+AYp3B52mSvYNJjRnxWYWr9avWKQ+i6lAuPvf3b2ju3/Hw1YQ6WRPogilhG973fIlTL8v6DRVdtGATmSkhXhypm55FanPYr1I/YvIRWozs3+Z2VzgtDhnS2xdT4Ijz4F3/wbbEut8frNGaXy3Twde/HAV2/YUBx1HRAIS6ymmH0YuUg8BsoArgHvilipZDLkDyoph2m1BJ6myUYNz2FVUygtzNZiQSH0Va4HYN8Lbd4BH3X0+0Ud9k/JadoXB18D8p6EwgJ5ma6BPdnP6ZDfjSQ0mJFJvxVog5pjZa4QLxBQzywTK4hcriZz4a2jSFl69AcoS65/s0rwcPl2/g5mf162hOETk0Ii1QPwPcCNwrLvvAtIJn2aSyjTIhNNvhVUF8NF/gk5TJd/t04GmGkxIpN6K9S6mMsLjQv/ezP4XOM7dE+9R4aD0HQEd+sPrt8LeHUGniVnD9BQuyu3E5IVrWb99T9BxROQQi/UupnuAXxDuZmMx8HMzuzuewZJKKAT5f4Lta+D9e4NOUyUjB3WmpMyZoMGEROqdWE8xfQc4090fcfdHgHzg7PjFSkKdB0Hvi+CDf8DmxDllc1hWE044vDVPzfySktLEuoYiIjVTld5cm5d73ay2g9QLZ/wRMJh6S8BBqubSvBxWb93DGx9rMCGR+iTWAnE38KGZPWZmjwNzgLviFytJNcuGE66DxS/CF+8HnSZmZxzVhnZNM3hS3YCL1CuxXqR+GsgDno9Mg919fDyDJa3jfgZNs2HyDVCWGF1qp6aEGDGwM+98soEvvtoZdBwROUQOWiDM7Jh9E9AeKARWAh0iy6Sq0hvBkNGw9iP48Mmg08TskoGdSA0ZT83SUYRIfZFayfq/HmSdo/6YqqfXMJj1EEwbDb3Oh4y6f0mnbdMMhvRqy4SClfzqzB5kpKUEHUlE4uygRxDufupBJhWH6jIL9/a6ayO885eg08Ts0rwctuwqZtKCxOp8UESqp7IjCADMbFiUxVuBj9xdt7ZUR4f+0H8kzPgnDLgCWnULOlGlBh/Wim5ZjXlyxgouHJAddBwRibOqdLXxMDAyMj0E/Ap438w09Gh1nXYLpGbAlJuDThITM+PSvBzmrdzCwlVbg44jInEWa4EoA45y9wvc/QKgJ7AXGATcEK9wSS+zLZz0G/jkVVg2Leg0MRl2TDYN01LUP5NIPRBrgeji7uvKza8Herj7JkAjytRE3tXQoitM+R2UlgSdplLNGqZxXr8OvDhvFVt361cvksxiLRDvmtkkM7vMzC4DJhIem7oxsCV+8eqB1AYw9E7Y8DEUPBJ0mphcmpfDnuIynp9bGHQUEYmjWAvENcCjQD+gP/A4cI2773T3A45NbWb5ZrbUzJaZ2Y1R1l9uZhvMbF5k+lG5daXllk+s2m4lmCO+A11PhjfvhF11f+yFozs2o1+n5jyhwYREklqsT1I78B7wBvA68I5X8s1gZinAGOAswtcsRphZzyhNn3H3fpHp4XLLd5dbfm4sOROWGeTfA3u3wVuJ0UnuqLwclm/YyfTPNgYdRUTiJNbuvi8GZgEXAhcDM83swko2Gwgsc/fl7l4EjAfOq0nYpNa2J+T+EGb/C9YvCTpNpc7u057mjdJ4cqYuVoskq1hPMd1MeDS5y9z9B4S//P9QyTYdCXfLsU9hZFlFF5jZAjN71sw6lVueYWYFZjbDzM6P9gFmdmWkTcGGDRti3JU67NSbwyPQTb4J6vipm4y0FC7O7cSURetYt02DCYkko1gLRKjCA3EbY9jWoiyr+K33EuE7pPoQPnX1eLl1nd09F/g+cK+Z7fckmbs/6O657p6blZVV6U7UeY1awik3wfI3YemrQaep1MhBnSktc8bP0mBCIsko1gIx2cymRC4qXw68DLxSyTaFQPkjgmxgdfkG7r7R3fdGZh8CBpRbtzrycznwFuGL48nv2P+B1kfAazdDyd7K2wcop1VjTuqRxVOzVlCswYREkk6sF6mvBx4E+gB9gQfdvbIH5GYD3c2sq5mlA5cQvj32a2bWvtzsucCSyPIWZtYg8ro1cDzhoU6TX0oa5N8Fm5bDzH8GnaZSo/JyWLdtL9OWrKu8sYgklJj6YgJw9+eA56rQvsTMrgWmACnAI+6+yMxGAwXuPpHw2NbnAiXAJuDyyOZHAWPNrIxwEbvH3etHgQA4/AzokQ9v/wX6joAmbYJOdECnHdmGDs0yeHLGl+Qf3b7yDUQkYdjB7lY1s+3sf90AwtcX3N2bxitYVeXm5npBQUHQMWrPV8vg/jzoewmcd1/QaQ7qvjc+5X9f+4Q3fn0yh2U1CTqOiFSBmc2JXO/dT2XdfWe6e9MoU2ZdKg5JqfXhMOgn4UGFVs8LOs1BXXxsJ9JSjHEaklQkqcR6kVqCcPJvoVErmHxjnb7ttU1mBkN7teM/BSvZXZQYw6iKSOVUIOqyjGZw+h/gy+mw6IWg0xzUqLwctu0p4aUFqytvLCIJQQWirus/Ctr2hqm3QPHuoNMc0MCuLenRtom6ARdJIioQdV0oBc66B7auhA/+EXSaA9o3mNCCwq3MX6kOfkWSgQpEIuhyAvQ8D977G2xdFXSaA/pe/440StdgQiLJQgUiUZx5O5SVwut/DDrJAWVmpHF+/45MnL+aLbuKgo4jIjWkApEoWuTAcT+DjybAylnR2zx6dngK0KWDcthbUsazczSYkEiiU4FIJCdcB5nt4dUboKxu9n3Us0NTBuS0YNzMLykrq7u35opI5VQgEkmDJnDGH2H1XFjwTNBpDmhUXg6ff7WT9z/7KugoIlIDKhCJpvfF0HFA+FrE3h1Bp4nqrN7taNk4XRerRRKcCkSiCYUg/0+wYy28939Bp4mqQWp4MKGpi9exZmvdfXZDRA5OBSIRdToW+gyHD+6DzV8EnSaqkYM648DTGkxIJGGpQCSqM/4YfojutcpGfg1Gp5aNOKVHFuNnfanBhEQSlApEomraAU74FSyZCJ+/G3SaqEYNzmH99r1MXazBhEQSkQpEIjvuWmjWOdzba1nd60X15B5t6Ni8IU9M18VqkUSkApHI0hrCkNGwbiHMfTzoNPtJCRkj8zozfflGlq3fHnQcEakiFYhE1/N8yDke3rgDykqCTrOfi3M7kZ4S4skZGkxIJNGoQCQ6M8i/G3Ztgi1170u4dZMGnNW7Hc/NKWRXUd0rYCJyYCoQyaB9XzjmB7B9DRTvCjrNfkbl5bB9bwkT52kwIZFEogKRLE77A1gINn1e54YnHZDTgiPbZfLv6SvwOpZNRA5MBSJZNMmC5p1hz2Z49Duw9qOgE31t32BCi9ds40MNJiSSMFQgkklmB2h5OHy1FMaeBJN+Fb42UQec378jjTWYkEhCUYFIJmaQ2Q5+NgcGXglzHoN/HAOzHw78OYkmDVIZdkw2kxasYfNODSaUqBbddQKL7joh6Bg1liz7AfHdFxWIZNSwBZz1J7jqPWh7NLz8axh7Mqz4INBYl+blUFRSxn/m1K/+mZLpy0jqFxWIZNa2J1z2Elz0OOzZAo+eBc/+D2wL5m6iI9plMrBLS56cocGERBKBCkSyM4Ne58M1s+DkG2DJS/CPXHj3r1Cy95DHuXRwDl9u2sU7n244aLvhY6czfOz0Q5RKRKJRgagv0hvBqb+Da2dBt1Nh2mgYMwiWTj6kMfJ7taN1k/RKn6y+ZeP13LLx+kOUSkSiUYGob1p0gUvGwagXICUNnh4O4y6Cr5Ydko9PTw0x/NhOvPHxOlZt0WBCInWZCkR91e00uPoDGHInrJgO9+fB1Fthb/w71RsxsDMAT8+se12DiMg34logzCzfzJaa2TIzuzHK+svNbIOZzYtMPyq37jIz+zQyXRbPnPVWSlq4y/CfzYE+F8P794avTyyYENensbNbNOK0I9swfvaXFJVoMCGRuipuBcLMUoAxwFlAT2CEmfWM0vQZd+8XmR6ObNsSuBUYBAwEbjWzFvHKWu9ltoXz74cfTQsPRPT8j+GRfFgzP24feWleDl/tKGLKorVx+wwRqZl4HkEMBJa5+3J3LwLGA+fFuO1QYKq7b3L3zcBUID9OOWWf7NxwkTj3Pti4LPzsxEu/hJ0ba/2jTuqeReeWjXhCT1aL1FnxLBAdgfJPRBVGllV0gZktMLNnzaxTVbY1syvNrMDMCjZsOPhtkxKjUAiOGRU+7ZR3Ncz9d/hp7FkPQWntddcdChkjB3Vm1uebWLpWgwmJ1EXxLBAWZVnFE9svAV3cvQ/wOrBvWLRYtsXdH3T3XHfPzcrKqlFYqaBh8/A4E1d/EO5O/JXfwIMnwxfv1dpHXJTbifTUEONm6ihCpC6KZ4EoBDqVm88GvvUIr7tvdPd9T2s9BAyIdVs5RNocCT/4L1z8BOzZBo+dDf+5ArYW1vitWzZO55ze7Xl+7ip27tVgQiJ1TTwLxGygu5l1NbN04BJgYozDMdEAAAzMSURBVPkGZta+3Oy5wJLI6ynAEDNrEbk4PSSyTA7mipfDU20zg57nwjUz4ZSbYOkrcN+x8M5foHhPjd760sE57NhbwovzVtVSWBGpLXErEO5eAlxL+It9CTDB3ReZ2WgzOzfS7OdmtsjM5gM/By6PbLsJuJ1wkZkNjI4skyClN4JTbgx323H4GeFxsO8fBB+/Uu3bYvt3ak7P9k15QoMJidQ5cX0Owt1fcfce7t7N3e+MLLvF3SdGXt/k7r3cva+7n+ruH5fb9hF3PzwyPRrPnFJFLXJg+BMw6kVIzYDxI2DchfDVp1V+KzNj1OAcPl67nTkrNschrIhUl56klurrdmq4S/Ghd8PKWeGnsV/7ffhaRRWc168DmQ1SNZiQSB2jAiE1k5IGg38KP5sLfUfAB/fBfbkwfzyUxfaUdKP0VC4YkM0rH61l445D38OsiESnAiG1o0kWnHcf/HgaNMuGF34CjwyF1R/GtPmleZ0pKi1jQkHN744SkdqhAiG1q+MA+J/X4bz7YfPn8OCpMPHnsPOrg252eJtM8g5rybiZKyjVYEIidYIKhNS+UAj6jww/jT34Gpg3Lvw09syxB30ae1ReFwo37+adT/RUvEhdoAIh8ZPRDIbeGX4au8Mx8OpvYeyJ8Pk7UZsP6dWWrMwG6p9JpI5QgZD4yzoiPEDR8HFQtAMe/y5MuAy2rPxWs7SUECOO7cSbS9eztqxZQGFFZB8VCDk0zOCoc8IP2Z16M3wyJfw09tt//tbT2CMGdSZkxuSifgGGFRFQgZBDLa0hnPzb8NjYPYbCm3fCmIGwZBK4075ZQ04/sg1TivtR7ClBpxWp11QgJBjNO8PFj8MPJkJaI3hmJDw5DDZ8wqjBOWzzRrxfckTQKUXqtdSgA0g9d9jJ4aexZz8Mb94FDwzmhIFXcXgohwd2D2HKfe+RkZZCw31Teso38+khGqZF5tO/aZOR/u32FdukhKL1Ji8iFalASPBSUiHvKuh9IUy7DZsxhpfS03m1bBAlJd0p2gt7y4yiUmdvKewtNYpKYWup85WHKMNwjNKvX4coJfw6PIXK/QwRCoVITU0hLTWVtNRUUlNTSUtLJT01PKVGXjdISyU9LY20tFQapKaSnpZKg/Q0Guz7mZ5KRloa6WlpZDRIIyMtlYbp6WQ0SKVBWhoWSgELfdORYUkR4JH5aD/LKizjgG3cyyhzx8vCP8vcocxxyigrc8q8NDwfWedlZXhkOy/X7uvlhJeXefgz3P1b82VeBmXOxpIMAJbNnhqOF+XX6f7tJ+ijDORScUGF1ZU/B1Oxyf7b7P8e5ZusLW4UXvbexP3aJZp1xY1Isfg8O2TJ0oNmbm6uFxQUBB1DasOquex6+Gwa+a6gk4gkhCVlnTlq9EfV2tbM5rh7brR1OoKQuqfjMXye1g3zMnpe/3rkr+bSb/7CLtv3uqzCurJw/09Rl5eGt426vGz/qdxyLyuluLSU4uISioqLKS4poaiklJKSYopLSikuKaGkpISSktLwz9LS8OvSEkpLS9ixagmOkdG2B2YWvqOL8M9v5kPR12FgocjiUHj9vvnINuW3C89/u92+edvv8755X/vW+vLblV8XYsM7/8Jx2p7y469/XRZtAEg7+Gk8s29f/tyvdbS3rLiwwmfst0mUU4n73mPN1P8HQPshvzxozkSw9rV7SUkt46g4vLcKhNRZbqHwGBQBMyA9MjWuxvaL7joBgF7X/LUWUwVj0Yy/AdDrlGEBJ6mht+4CoNfgswIOUgvevDNub627mEREJCoVCBERiUoFQkREolKBEBGRqFQgREQkKhUIERGJSgVCRESiUoEQEZGoVCBERCQqFQgREYlKBUJERKJSgRARkahUIEREJCoVCBERiUoFQkREooprgTCzfDNbambLzOzGg7S70MzczHIj813MbLeZzYtM/4xnThER2V/cBgwysxRgDHAmUAjMNrOJ7r64QrtM4OfAzApv8Zm794tXPhERObh4HkEMBJa5+3J3LwLGA+dFaXc78GdgTxyziIhIFcWzQHQEVpabL4ws+5qZ9Qc6ufukKNt3NbMPzextMzsx2geY2ZVmVmBmBRs2bKi14CIiEt8CEW3Ucv96ZXjU8r8Bv47Sbg3Q2d37A78CnjKzpvu9mfuD7p7r7rlZWVm1FFtERCCO1yAIHzF0KjefDawuN58JHA28ZWYA7YCJZnauuxcAewHcfY6ZfQb0AArimFfqkF7tmwUdQaTei+cRxGygu5l1NbN04BJg4r6V7r7V3Vu7exd37wLMAM519wIzy4pc5MbMDgO6A8vjmFVERCqI2xGEu5eY2bXAFCAFeMTdF5nZaKDA3SceZPOTgNFmVgKUAle5+6Z4ZZU66IqXg04gUu/F8xQT7v4K8EqFZbccoO0p5V4/BzwXz2wiInJwepJaRESiUoEQEZGozN0rb5UAcnNzvaBANzmJiFSFmc1x99xo63QEISIiUalAiIhIVCoQIiISlQqEiIhEpQIhIiJRqUCIiEhUKhAiIhKVCoSIiESlAiEiIlElzZPUZrYBWFGDt2gNfFVLcYKULPsB2pe6Kln2JVn2A2q2LznuHnXEtaQpEDVlZgUHetw8kSTLfoD2pa5Kln1Jlv2A+O2LTjGJiEhUKhAiIhKVCsQ3Hgw6QC1Jlv0A7UtdlSz7kiz7AXHaF12DEBGRqHQEISIiUalAiIhIVCoQEWZ2u5ktMLN5ZvaamXUIOlN1mdlfzOzjyP68YGbNg85UXWZ2kZktMrMyM0u4WxLNLN/MlprZMjO7Meg8NWFmj5jZejNbGHSWmjCzTmb2ppktify39YugM1WXmWWY2Swzmx/Zl9tq9f11DSLMzJq6+7bI658DPd39qoBjVYuZDQHecPcSM/sTgLvfEHCsajGzo4AyYCzwG3dPmHFlzSwF+AQ4EygEZgMj3H1xoMGqycxOAnYA/3b3o4POU11m1h5o7+5zzSwTmAOcn4i/FzMzoLG77zCzNOA94BfuPqM23l9HEBH7ikNEYyBhK6e7v+buJZHZGUB2kHlqwt2XuPvSoHNU00Bgmbsvd/ciYDxwXsCZqs3d3wE2BZ2jptx9jbvPjbzeDiwBOgabqno8bEdkNi0y1dp3lwpEOWZ2p5mtBEYCtwSdp5b8EHg16BD1VEdgZbn5QhL0iyhZmVkXoD8wM9gk1WdmKWY2D1gPTHX3WtuXelUgzOx1M1sYZToPwN1vdvdOwDjg2mDTHlxl+xJpczNQQnh/6qxY9iVBWZRlCXtkmmzMrAnwHPDLCmcQEoq7l7p7P8JnCgaaWa2d/kutrTdKBO5+RoxNnwJeBm6NY5waqWxfzOwy4BzgdK/jF5qq8HtJNIVAp3Lz2cDqgLJIOZHz9c8B49z9+aDz1AZ332JmbwH5QK3cSFCvjiAOxsy6l5s9F/g4qCw1ZWb5wA3Aue6+K+g89dhsoLuZdTWzdOASYGLAmeq9yIXdfwFL3P3/gs5TE2aWte8uRTNrCJxBLX536S6mCDN7DjiC8B0zK4Cr3H1VsKmqx8yWAQ2AjZFFMxL4jqzvAf8AsoAtwDx3HxpsqtiZ2XeAe4EU4BF3vzPgSNVmZk8DpxDuWnodcKu7/yvQUNVgZicA7wIfEf7/HeB37v5KcKmqx8z6AI8T/u8rBExw99G19v4qECIiEo1OMYmISFQqECIiEpUKhIiIRKUCISIiUalAiIhIVCoQIlVgZjsqb3XQ7Z81s8Mir5uY2Vgz+yzSE+c7ZjbIzNIjr+vVg6xS96hAiBwiZtYLSHH35ZFFDxPu/K67u/cCLgdaRzr2mwYMDySoSIQKhEg1WNhfIn1GfWRmwyPLQ2Z2f+SIYJKZvWJmF0Y2Gwn8N9KuGzAI+L27lwFEen19OdL2xUh7kcDoEFakeoYB/YC+hJ8snm1m7wDHA12A3kAbwl1JPxLZ5njg6cjrXoSfCi89wPsvBI6NS3KRGOkIQqR6TgCejvSkuQ54m/AX+gnAf9y9zN3XAm+W26Y9sCGWN48UjqLIgDYigVCBEKmeaF15H2w5wG4gI/J6EdDXzA72/2ADYE81sonUChUIkep5BxgeGawlCzgJmEV4yMcLItci2hLu3G6fJcDhAO7+GVAA3BbpXRQz675vDAwzawVscPfiQ7VDIhWpQIhUzwvAAmA+8Abw28gppecIjwOxkPA42jOBrZFtXubbBeNHQDtgmZl9BDzEN+NFnAokXO+iklzUm6tILTOzJpFB5FsRPqo43t3XRvrrfzMyf6CL0/ve43ngpgQej1uSgO5iEql9kyKDuKQDt0eOLHD33WZ2K+Fxqb880MaRwYVeVHGQoOkIQkREotI1CBERiUoFQkREolKBEBGRqFQgREQkKhUIERGJ6v8DoKiAyrq6PpIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = -grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ] \n",
    "# train_means = -grid.cv_results_[ 'mean_train_score' ]    #  查看上面数据没有mean_train_score相关数据\n",
    "# train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores =  np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "#train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "#train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    #pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'logloss' )\n",
    "pyplot.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 按正确率做评价"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7747395833333334\n",
      "{'C': 0.1, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "\n",
    "#缺省scoring为正确率\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5)\n",
    "grid.fit(X_train,y_train)\n",
    "\n",
    "print(grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXwV9b3/8dcnISHsS1hkJ0BQFgE1gopLwY2qxeJSodW2tlfsr+r1eqtWba8LVlv1tra99d661FqtFRFcUFDcd0UQWcMWQCCAEHbCFpJ8fn/MiYZwAodwJpPl/Xw8ziMzc74z5zMRzyffz8x8v+buiIiIVJQSdQAiIlIzKUGIiEhcShAiIhKXEoSIiMSlBCEiInEpQYiISFyhJggzG2Fmi80sz8xuifN+VzN7x8y+MLO5ZnZeufduje232MzODTNOERE5kIX1HISZpQJLgLOBfGAGMMbdc8u1eQT4wt3/z8z6AlPdvXts+RlgMNAReBPo7e4loQQrIiIHCLMHMRjIc/fl7l4EjAcurNDGgeax5RbA2tjyhcB4d9/r7iuAvNjxRESkmjQI8didgNXl1vOBIRXa3Am8bmbXAU2As8rt+2mFfTsd7MPatGnj3bt3P4JwRUTqn88//3yju7eN916YCcLibKtYzxoDPOHuvzezk4GnzKx/gvtiZmOBsQBdu3Zl5syZRxiyiEj9YmYrK3svzBJTPtCl3HpnvikhlfkpMAHA3T8BMoA2Ce6Luz/i7jnuntO2bdwEKCIiVRRmgpgBZJtZlpmlA6OByRXarALOBDCzPgQJoiDWbrSZNTSzLCAb+CzEWEVEpILQSkzuXmxm1wLTgFTgcXdfYGbjgJnuPhn4BfComd1AUEL6sQe3VS0wswlALlAMXKM7mEREqldot7lWt5ycHNc1CBEps2/fPvLz89mzZ0/UodQIGRkZdO7cmbS0tP22m9nn7p4Tb58wL1KLiEQmPz+fZs2a0b17d8zi3fdSf7g7mzZtIj8/n6ysrIT301AbIlIn7dmzh8zMzHqfHADMjMzMzMPuTSlBiEidpeTwjar8LpQgRERiLnv4Ey57+JOow6gxlCDqkr+fH7zqgrp0LlJvNW3a9OvlESNG0LJlSy644IK4ba+55hoGDRpE3759adSoEYMGDWLQoEFMnDjxsD5z1qxZvPbaa0cUdxldpBYRqQY33XQTu3bt4uGHH477/kMPPQTAl19+yQUXXMDs2bOr9DmzZs1i/vz5jBgxosqxllEPQkSkGpx55pk0a9asSvsuXbqUc889lxNOOIHTTz+dJUuWADB+/Hj69+/PwIEDGTZsGLt372bcuHE8/fTTVep9VKQehIjUeXe9vIDctdsP2S53XdAmkesQfTs2547v9Dvi2BIxduxYHnvsMXr27MlHH33Etddey+uvv85dd93Fu+++S/v27dm6dSuNGjXi9ttvZ/78+fzxj3884s9VghAJW9m1lCunRBtHEiy491QA+t32YcSRHJnSol0ApKQ3jjiSQ9u6dSuffvopF1988dfbiouLARg6dCiXX3Yxoy44l9E/uSbpn60EIRKyBeu2AVA9f2tKPBX/0t+9bhEAjTocs9/2sp7Ds1efXD2BJcDdadOmTdxrEo8++ijvT3mWqW++y8CBA5k7d25SP1vXIEREarBWrVrRoUMHXnjhBQBKS0uZM2cOAMuXL2fwCYO44+bradWqFWvWrKFZs2bs2LEjKZ+tBCEiUg1OO+00Lr30Ut566y06d+7MtGnTEt53/Pjx/PWvf2XgwIH069ePV155BYAbbriBE4eP5MThIznrrLPo378/w4cPZ86cORx33HG6SC0iUlMVFhZ+vfzBBx8ktE/37t2ZP3/+ftt69OgRN6FMnjz5gHJZ27ZtkzZ5mhKEiEhMTbr2UBOoxCQiInEpQYiISFxKECIiEpcShIiIxKUEISJSRqMI7yfUBGFmI8xssZnlmdktcd5/0Mxmx15LzGxruffuN7MFZrbQzP5smvlDRGqZsuG+Z8+ezcknn0y/fv0YMGAAzz777AFt69Vw32aWCjwEnA3kAzPMbLK755a1cfcbyrW/DjgutnwKMBQYEHv7Q+AM4N2w4hURCUvjxo158sknyc7OZu3atZxwwgmce+65tGzZ8us29W2478FAnrsvd/ciYDxw4UHajwGeiS07kAGkAw2BNGB9iLGKiISmd+/eZGdnA9CxY0fatWtHQUFBwvsfbLjvnGHfYchZ3611w313AlaXW88HhsRraGbdgCzgbQB3/8TM3gHWAQb8xd0XxtlvLDAWoGvXrkkNXkTqkFdvga/mfb2aHhvNlYqjuX4VG+wukesQRx0L3/7dYYfy2WefUVRURM+ePRPe52DDfb864e+0b9uGvY2OqlXDfce7ZuCVtB0NTHT3EgAz6wX0ATrH3n/DzE539/f3O5j7I8AjADk5OZUdW0SkRli3bh1XXHEF//jHP0hJSayAc6jhvq/691tq5XDf+UCXcuudgbWVtB0NlD+7UcCn7l4IYGavAicB78fZV0Tk4Cr8pV9UyXDfYc7dsX37ds4//3x+85vfcNJJJyW8X10d7nsGkG1mWWaWTpAEJldsZGZHA62A8lM4rQLOMLMGZpZGcIH6gBKT1F0L1m37eh4FkdquqKiIUaNG8cMf/pBLL730sPatk8N9u3sxcC0wjeDLfYK7LzCzcWY2slzTMcB4dy9fIpoILAPmAXOAOe7+clixioiEacKECbz//vs88cQTX9++ejh3KdXJ4b7dfSowtcK22yus3xlnvxLg6jBjExEJW9lw35dffjmXX355QvtouG+RQzAvJZUSKC2BlNSow5H6og7MG55MShBSc5SWwJcfwLznOHrfQlIphbvbQNP20KwDNO8Y+9kBmnXc/2fDZlFHL1LnKEFItNxh7SyYNxHmT4LC9ZDejO0pLdhjjegw9AewfR3sWAublgUJZE+ci9fpzWIJo3wiqZBQmrZTb6SecXc0Sk9g/8u8iVGCkGhsXArzngtem5dDajr0PheOvRSyz2HtA2cD0GH4rw/ct2gX7FgH29fG/7niAyj8CkqL99/PUoPeSKWJJPazYdNq+AVI2DIyMti0aROZmZn1Pkm4O5s2bSIjI+Ow9lOCkOqzfW3QS5j3HKybA5YCWafDab+AYy6ARi0PfQwInn7N7Bm8KlNaCjsLgp5HWQ9k+7pvEsmmvCCR7I3TG2nYPE4pq0IiadL2kL2R3UUlzFq1hSl7TmU36XSatpiMtBQy0lJpmJZKRoNgOXjFlhuklmvzzba0VKv3X3KHq3PnzuTn58cd0mLftq8ASNta+5+vTfRcMjIy6Ny580HbVKQEIeHatRkWTg5KSF9+CDh0OgFG/A76jYJmR4XzuSkp0Kx98Op4XOXt9hbCjq/iJJLYz43vBe8HD/mXO36DctdGgkSyt3F7lu9tzpytjfhwQzrvf9WA7SUNSWEo6RRT9G4epVX8PkoxvkkmDcolmbSU/ZJKWbJp2KBi4qnwfoVklLHfsVJp2CCFlJTanZDS0tLIysqK+96Ce68CoM9tH1ZnSKEI81yUICT5inbBkleDpLD0DSjdB5nZMOw26H/xwf/yr24Nm0LDXtCmV+VtSkugcMMBvZCirWsoLFhF6Yq5NN77Jo19N30IxogZDZAG+xo3p3hfEW5Go459cQ+6+6XulH69TNz1/ZfL2sSWi53SfRX2LwWn3D6lXunYNuXtjb0q9qXMIMWCnktKbDll3x4MZ8lvTqza77uGSC3aDVDrzwOCc9lIgr3vw6QEAaE+Xl9vlOyD5e8G5aNFU6CoMCjPnPSz4LrCUQOCb5zaKCUVmndga4NMPtvaiekbNjN9xSZy126n1CE9NYWBXVpwatcMhrYron+znWTsDhJK2vZ17J75LEYp1qjV1wOUVdel8rLEUuJOaek3yaOk9JskVVoaez+WZOK9X+pOcSns27cFBxo0aFFNZxCO4qIgddb284DgXIotnK9yJQiputJSyP8sSAoLXoBdmyCjZdBLOPZS6DY0KPXUUpsK9/LZis1MX7GZT5dvYvH6HbhDwwYpHNe1JdcNz2ZIj9Yc37UVGWmVf+WvnvMpAP0un1RdoX/NCJJRshLSgntPBaDfLW8m6YjRqCvnAcG5tKX40A2rQAlCDt/6BbE7kCbBtlXQoBEc/e0gKfQ6Exo0jDrCKtmwYw/Tlwe9g+nLN7N0Q/AUbKO0VE7o1orzj+3AkB6ZDOzSgoYNdLus1H1KEJKYLSth/sTgusKG3OCW0Z7DYfiv4ZjzauWDauu27d4vISzfuBOAJump5HRvzajjOzEkK5NjO7UgvUHt7QmJVJUShFSusAByXwx6C6unB9u6nATn/XdwB1KTNtHGd5hWb97F9BWbmb58E9NXbGbV5mDSmGYZDRjcvTWjB3dhSFYm/To2p0GqEoKIEoTsb++O4CLzvOdg2TvB7Z3t+sGZdwTXFlp1izrChLg7Kzft+rp3MH3FZtZsDe5cadk4jcHdW/OjU7ozJKs1fTo0J7WW39IpEgYlCIHivZD3ZpAUFr8GxbuhRVcYej0cewm07xd1hIfk7iwr2FkuIWxi/fa9AGQ2SWdIj9aMPb0HQ3q0pne7ZrX+Hn+R6qAEUV+VlsDKj4KkkPtSML5R40w47vLgYnOXwTX6ttTSUmfphsL9eggbC4OE0K5ZQ4b0yGRIVmtO6tGanm2b6ilkkSpQgqhP3IMhLuY9B/OfDx78Sm8aDHNx7KXQ4wxITYs6yrhKS52FX23/unfw2YrNbNm1D4COLTI4LbsNQ7JaM6RHJt0zGyshiCSBEkR9sGlZcPfRvOdg01JISYPsc+DYe6D3iGBsoxqm1GFZ6VF8/P7yrxPC9j3Bvd5dWjfizD7tYz2ETDq3aqSEIBICJYi6asdXQS9h3nPBcNoYdD8VTrkO+o6ERq2ijvCgfr97JO8W94OpC8lq04Tzju3AkB6tGZKVSceWjaIOT6ReCDVBmNkI4E8ED3I+5u6/q/D+g8Cw2GpjoJ27t4y91xV4DOgCOHCeu38ZZry1Xmkx7NoI/xgJK94HHDoMgnPugf4XBaOQ1gJT5q7j3eJ+jEqfzi033kb75oc3RLGIJEdoCcLMUoGHgLOBfGCGmU1299yyNu5+Q7n21wHlh918ErjH3d8ws6ZAaVix1mrusOoTmPVUMOyFlwavM34Z3IHUJjvqCA/LpsK93P7SfLJT1nFlw3do33xc1CGJ1Fth9iAGA3nuvhzAzMYDFwK5lbQfA9wRa9sXaODubwC4e2GIcdZOO9bDnH/BF/8M5jZIbwZN2gVDUF/9Xo2+A+lg7nw5l+179jGu0SukWu0fq1+kNgszQXQCVpdbzweGxGtoZt2ALODt2KbewFYzez62/U3gFveKg/LXMyXFsPR1+OIpWDIteIit68lw6n9Cv+/C098L2tXS5PDa/K94ec5afnF2b7pP3xh1OCL1XpgJIt63VGV/Eo4GJpZLAA2A0whKTquAZ4EfA3/b7wPMxgJjAbp27XrkEddUG/OCpDDnmWDO5ibt4JRr4bgral0JqTJbdhbx6xfn069jc372rZ4smR51RCISZoLIJ7jAXKYzsLaStqOBayrs+0W58tSLwElUSBDu/gjwCEBOTk7dqkcU7QweYJv1FKz6OBgcr/e5wYNs2efU2OcVqmrcK7ls3VXEkz8ZTJrGQRKpEcJMEDOAbDPLAtYQJIHvV2xkZkcDrYBPKuzbyszaunsBMByYGWKsNYM7rJkFXzwZDKVdtANa94Sz7oSBY8KbnjNib+au54Uv1nD9mdn07dg86nBEJCa0BOHuxWZ2LTCN4DbXx919gZmNA2a6++RY0zHAeHf3cvuWmNmNwFsWPAH1OfBoWLFGbucmmPtsUEbakBvMr9Dvu0EJqdsptfaaQiK27drHbS/M45ijmnHNsINM+yki1S7U5yDcfSowtcK22yus31nJvm8AA0ILLmqlJbD8naCEtHgqlBRBx+PhggeDUVMzav9UiIm4e0oum3YW8fiPT9ScCyI1jJ6krm5bVsLsp+GLp2F7fvBEc85P4fgrasWoqcn0zuINTPw8n2uH9aJ/p/0T4rjMB4Dg7gQRiYYSRHXYtwcWvRKUkJa/F2zrOQzOuRuOOb/WTtF5JLbv2cetk+bRu31TrjtTpSWRmkgJIkxfzQtKSPMmwO4twRwL37oFBn0fWtbh23ITcO+UhWzYsYeHrxiq+Z1FaigliGTbvTWYu3nWU7BuNqSmB8NpH38FZH0LUlRnf39JAeNnrOZnZ/RkYJeWUYcTun4d6sf1JKl7lCCSwR2+/DAoIeW+BMV7oH1/GHEfDPgeNG4ddYQ1RuHeYm59fh492zbhP86qGw/5idRVShBHYvu62AXnf8KWFdCweVA+Ou4K6Hhcnb49tap+O3Uha7ftZuLPTiEjTaUlkZpMCeJwleyDJa8FJaS8N4KRU7udGlxb6DOyRk6+U1N8nLeRp6ev4qrTsjihW82ej0JElCASV7AkeMJ5znjYWQBNj4Kh1we9hcyeUUdX4+3cW8zNk+aS1aYJvzjn6KjDqV5XTok6gqTpd9uHUYeQFHXlPCDcc1GCOJi9hbDgheDawurpsfGQRgQXnHudDan69SXq/tcWsWbrbiZcfbJKSyK1hL7hKnKH/Bkw68kgORQVQmYvOOuu2HhI7aOOsNb5dPkm/vHJSq4c2p0Tu+uCvUhtoQRRpmQffPyXoLdQsAjSGkO/UUEJqetJuuBcRbuLSvjlpLl0bd2Ym86tZ6UlkVpOCWL7OihYCLs2Q/506JQD3/kT9LsIMjSy6JF6YNpiVm7axTNXnUTj9MT/uT179ckhRiUiiVCCaNQymHuhWQe44nlo1yfqiOqMmV9u5u8fr+CKk7pxcs/MqMMRkcOkBJHWCDqeEJSQlBySZs++Em6eOJdOLRtxy7ePiTocEakCJQjQ9YUQ/OGNJSzfuJOn/20ITRrqn5lIbaSBgSTpZq3awmMfLGfM4K4M7dUm6nBEpIqUICSp9uwr4abn5nBU8wxuO0+lJZHaTH1/Sao/vbWUZQU7+cdPBtMsIy3qcETkCITagzCzEWa22MzyzOyWOO8/aGazY68lZra1wvvNzWyNmf0lzDglOeas3srD7y3jezmdOaN326jDEZEjFFoPwsxSgYeAs4F8YIaZTXb33LI27n5DufbXAcdVOMzdwHthxSjJs7e4hJsmzqFdswx+dX7fqMMRkSQIswcxGMhz9+XuXgSMBy48SPsxwDNlK2Z2AtAeeD3EGCVJ/vJ2HkvWF/Lbi46lRSOVlkTqgjATRCdgdbn1/Ni2A5hZNyALeDu2ngL8HrgpxPgkSeav2cb/vruMi47vxLBj2kUdjogkSZgJIt7DBV5J29HARHcvia3/HJjq7qsraR98gNlYM5tpZjMLCgqOIFSpqqLiUm6aOJfMJunccUG/qMMRkSQK8y6mfKBLufXOwNpK2o4Grim3fjJwmpn9HGgKpJtZobvvd6Hb3R8BHgHIycmpLPlIiP733TwWrtvOoz/MoUVjlZZE6pIwE8QMINvMsoA1BEng+xUbmdnRQCvgk7Jt7v6Dcu//GMipmBwkegvXbecvb+dx4aCOnN1Xw6CL1DWhlZjcvRi4FpgGLAQmuPsCMxtnZiPLNR0DjHd39QBqkX0lpdw0cQ4tG6dx53dUWhKpi0J9UM7dpwJTK2y7vcL6nYc4xhPAE0kOTY7Qw+8tY/6a7fz18uNp1SQ96nBEJAQaakMO25L1O/jzW3mcP6ADI/p3iDocEQmJEoQcluKSUm56bg5NMxowbqRKSyJ1mcZiksPy6AcrmJO/jb98/zgymzaMOhwRCVFCPQgz6x92IFLz5W0o5ME3lzCi31Gcf6xKSyJ1XaIlpr+a2Wdm9nMzaxlqRFIjlZQ6N02cQ+P0VO7+bn9MkyyJ1HkJJQh3PxX4AcGDbzPN7F9mdnaokUmN8viHK/hi1VbuGtmPts1UWhKpDxK+SO3uS4FfA78EzgD+bGaLzOyisIKTmmF5QSH//fpizurTnpEDO0YdjohUk0SvQQwwswcJHngbDnzH3fvElh8MMT6JWGmp88tJc2nYIIV7R6m0JFKfJHoX01+AR4Hb3H132UZ3X2tmvw4lMqkR/vHJl8z4cgu/v3Qg7ZpnRB2OiFSjQyaI2MQ/q939qXjvV7ZdInDllKQebuWmndz32iKGHd2Wi46PO1K7iNRhhywxxYbgzjQzjadQj5SWOjdPnEtaSgr3XnSsSksi9VCiJaaVwEdmNhnYWbbR3f8QSlRSJZc9HAyI++zVJx/xsZ6evpLpKzZz/8UD6NCi0REfT0Rqn0QTxNrYKwVoFl44UhOs3ryL3766iNN7t+XSnM5RhyMiEUkoQbj7XWEHIjWDu3PL83NJMeO3Ki2J1GsJJQgzawvcDPQDvr6Vxd2HhxSXROSZz1bzUd4m7hnVn04tVVoSqc8SfVDuaWARkAXcBXxJMGOc1CFrtu7m3qkLOaVnJt8f3DXqcEQkYokmiEx3/xuwz93fc/efACeFGJdUM3fnlklzKXXnvosHqLQkIglfpN4X+7nOzM4nuGBdd65eJvn5gdrouZn5fLB0I+Mu7EeX1o2jDkdEaoBEE8RvzKwF8Avgf4DmwA2hRSXVat223dz9Si5Dslpz+ZBuUYcjIjVEoncxvRJb3AYMS/TgZjYC+BOQCjzm7r+r8P6D5Y7XGGjn7i3NbBDwfwSJqAS4x92fTfRzJXHuzm3Pz6O41Ln/kgGkpKi0JCKBRO9i+jvgFbfHrkVUtk8q8BBwNpAPzDCzye6eW27/G8q1vw44Lra6C/ihuy81s47A52Y2zd23JhKvJO75WWt4Z3EBt1/Ql26ZTaIOR0RqkERLTK+UW84ARhFchziYwUCeuy8HMLPxwIVAbiXtxwB3ALj7krKNsQEBNwBtASWIJFq/fQ93vbyAE7u34sendI86HBGpYRItMU0qv25mzwBvHmK3TsDqcuv5wJB4Dc2sG8EttG/HeW8wkA4sSyRWSYy786sX5rO3uJT7Lxmo0pKIHCDhCYMqyAYOdaN8vG+cA8pUMaOBibGBAb85gFkH4CngSncvPeADzMaa2Uwzm1lQUJBA2FJm8py1vLlwPTeeczRZbVRaEpEDJTph0A4z2172Al4mmFnuYPIJpigt05nKy1KjgWcqfGZzYArwa3f/NN5O7v6Iu+e4e07btm0TORUBCnbs5Y7JCziua0t+cmpW1OGISA2VaImpKgP0zQCyzSwLWEOQBL5fsZGZHQ20Aj4pty0deAF40t2fq8JnSyXcnf96cT67ikp44JKBpKq0JCKVSLQHMSr2HETZeksz++7B9nH3YuBaYBrBVKUT3H2BmY0zs5Hlmo4Bxrt7+fLT94DTgR+b2ezYa1CC5yQHMWXeOl5b8BU3nNWbXu2aRh2OiNRgid7FdIe7v1C24u5bzewO4MWD7eTuU4GpFbbdXmH9zjj7/RP4Z4KxSYI2Fe7l9pcWMLBzC646TaUlETm4RC9Sx2uXaHKRGuL2yQso3FPM/ZcMpEFqVe9PEJH6ItFviZlm9gcz62lmPWJPQH8eZmCSXK/NX8eUuev49zN7cfRRmvNJRA4t0QRxHVAEPAtMAHYD14QVlCTXlp1F/PrF+fTr2Jyrz+gZdTgiUkskehfTTuCWkGORkNz58gK27trHUz8dQppKSyKSoETvYnrDzFqWW29lZtPCC0uS5Y3c9bw0ey3XDu9Fnw7Now5HRGqRRP+cbFN+oDx33wK0CyckSZatu4q47YV5HHNUM37+rV5RhyMitUyiCaLUzL4eWsPMulP5sBlSQ4x7JZfNO4v470sHkt5ApSUROTyJ3qr6K+BDM3svtn46MDackCQZ3l60nudnreG64b3o36nFoXcQEakg0YvUr5lZDkFSmA28RHAnk9RA23bv47bn59O7fVOuHa7SkohUTaITBv0bcD3BgHuzgZMIxk4aHl5oUlX3TMmloHAvj/zwBBo2SI06HBGppRItTF8PnAisdPdhBDO/aXztGmjrriImzMxn7Ok9GNC55aF3EBGpRKIJYo+77wEws4buvgg4OrywpCqKS50VG3fRq11Trj8zO+pwRKSWS/QidX7sOYgXgTfMbAuHnnJUqtnqzbsoKinl/ksGkJGm0pKIHJlEL1KPii3eaWbvAC2A10KLSg7bR3kb2bBjLx1aZHB811ZRhyMidcBhj8jq7u8dupVUp517i/nlpLlkpKXQuWWjqMMRkTpCT0/VAfe9tog1W3fTo00TUjRDnIgkiRJELffJsk08+clKrjwli2YZaVGHIyJ1iBJELbarKCgtdctszE3n6qYyEUmuUBOEmY0ws8VmlmdmBwwXbmYPlptzeomZbS333o/MbGns9aMw46ytHpi2mFWbd3HfxQNolK67lkQkuUKbNtTMUoGHgLOBfGCGmU1299yyNu5+Q7n21xE8gIeZtQbuAHIIBgX8PLbvlrDirW1mfLmZJz7+kh+d3I2TemRGHY6I1EFh9iAGA3nuvtzdi4DxwIUHaT8GeCa2fC7whrtvjiWFN4ARIcZaq+wuKuHmiXPp3KoRN484JupwRKSOCjNBdAJWl1vPj207gJl1A7KAtw933/roD28sZsXGndx30QCaNAytEygi9VyYCSLe/ZaVzSExGpjo7iWHs6+ZjTWzmWY2s6CgfgwN9fnKLfztwxX8YEhXTunVJupwRKQOCzNB5ANdyq13pvLhOUbzTXkp4X3d/RF3z3H3nLZt2x5huDXfnn0l3DxxDh1aNOLW8/pEHY6I1HFhJogZQLaZZZlZOkESmFyxkZkdDbQiGD68zDTgnNjc162Ac2Lb6rU/vrmUZQU7+e1Fx9JUpSURCVlo3zLuXmxm1xJ8sacCj7v7AjMbB8x097JkMQYY7+5ebt/NZnY3QZIBGOfum8OKtTaYs3orj7y/jMtyunB677rfWxKR6IX6Z6i7TwWmVth2e4X1OyvZ93Hg8dCCq0X2Fpdw43NzaN88g19doNKSiFQP1Slqgf95K4+lGwr5+5Un0lzDaYhINdFQGzXc/DXb+L/3lnHx8Z0ZdnS7qMMRkXpECaIGKyou5cbn5pDZJJ3bL+gbdTgiUgAnEjAAAA4VSURBVM+oxFSDPfROHou+2sFjP8yhRWOVlkSkeqkHUUPlrt3OQ+/k8d1BHTmrb/uowxGRekgJogbaVxKUllo2TueO7/SLOhwRqadUYqqB/vruMnLXbeevl59AqybpUYcjIvWUehA1zOKvdvDnt5dywYAOjOh/VNThiEg9pgRRgxSXlHLTxDk0z0jjrpEqLYlItFRiqkEe+WA5c/O38dD3jyezacOowxGRek49iBoib8MO/vjGUr7d/yjOH9Ah6nBERJQgaoKSUufG5+bSpGEq4y7sH3U4IiKASkw1wt8+XM7s1Vv50+hBtG2m0pKI1AzqQURseUEhv399CWf3bc/IgR2jDkdE5GtKEBEqKXVunjiXjLRU7vluf8zizbQqIhINlZgi9MTHXzJz5Rb+8L2BtGueccTHe/bqk5MQlYhIQD2IiHy5cScPTFvE8GPaMeq4TlGHIyJyACWICJSWOjdPmktaagr3jjpWpSURqZGUICLwz+kr+WzFZv7rgr4c1eLIS0siImEINUGY2QgzW2xmeWZ2SyVtvmdmuWa2wMz+VW77/bFtC83sz1ZH/sxevXkXv3t1Eaf3bsulJ3SOOhwRkUqFdpHazFKBh4CzgXxghplNdvfccm2ygVuBoe6+xczaxbafAgwFBsSafgicAbwbVrzVwd355aS5pJjxu4tUWhKRmi3MHsRgIM/dl7t7ETAeuLBCm6uAh9x9C4C7b4htdyADSAcaAmnA+hBjrRb/+mwVHy/bxG3n9aFjy0ZRhyMiclBhJohOwOpy6/mxbeX1Bnqb2Udm9qmZjQBw90+Ad4B1sdc0d19Y8QPMbKyZzTSzmQUFBaGcRLLkb9nFvVMWcmqvNowZ3CXqcEREDinMBBGvfuIV1hsA2cC3gDHAY2bW0sx6AX2AzgRJZbiZnX7Awdwfcfccd89p27ZtUoNPJnfn1ufn4cBvVVoSkVoizASRD5T/U7kzsDZOm5fcfZ+7rwAWEySMUcCn7l7o7oXAq8BJIcYaqgkzV/PB0o3c+u1j6NK6cdThiIgkJMwEMQPINrMsM0sHRgOTK7R5ERgGYGZtCEpOy4FVwBlm1sDM0gguUB9QYqoN1m3bzW9eWchJPVrzgyHdog5HRCRhoSUIdy8GrgWmEXy5T3D3BWY2zsxGxppNAzaZWS7BNYeb3H0TMBFYBswD5gBz3P3lsGINS1lpqbjUuf/igaSkqLQkIrVHqGMxuftUYGqFbbeXW3bgP2Ov8m1KgKvDjK06TJq1hncXF3DHd/rSNVOlJRGpXfQkdUjWb9/DuJcXMLh7a350cveowxEROWxKECFwd371wjz2Fpdy3yUDVFoSkVpJCSIEL81ey5sLN3DTuUeT1aZJ1OGIiFSJEkSSbdixhztfXsDxXVty5dCsqMMREakyJYgkcnf+68X57Coq4f5LBpKq0pKI1GJKEEn0ytx1TFuwnv88uze92jWNOhwRkSOiBJEkGwv3csfkBQzs3IJ/O1WlJRGp/ZQgkuSOlxZQuKeYBy4dSINU/VpFpPbTN1kSvDpvHVPmreP6s7Lp3b5Z1OGIiCSFEsQR2ryziP96aT79OzVn7Ok9og5HRCRpQh1qoz64c/ICtu3ex1M/HUKaSksiUofoG+0IvL7gKybPWcu1w7Lp06F51OGIiCSVEkQVbd1VxK9enE+fDs35+bCeUYcjIpJ0KjFV0bhXctmys4gnrjxRpSURqZP0zVYFby9az/Oz1vDzb/WkX8cWUYcjIhIKJYjDtG33Pm59fh5Ht2/GtcOzow5HRCQ0KjEdpnum5LKxsIhHf5hDegPlVxGpu/QNdxjeXbyBCTPzufr0Hgzo3DLqcEREQhVqgjCzEWa22MzyzOyWStp8z8xyzWyBmf2r3PauZva6mS2Mvd89zFgPZceeoLTUq11T/v1MlZZEpO4LrcRkZqnAQ8DZQD4ww8wmu3tuuTbZwK3AUHffYmbtyh3iSeAed3/DzJoCpWHFmoh7py5i/fY9TPp/p5CRlhplKCIi1SLMHsRgIM/dl7t7ETAeuLBCm6uAh9x9C4C7bwAws75AA3d/I7a90N13hRjrQX24dCPPfLaKq07rwXFdW0UVhohItQozQXQCVpdbz49tK6830NvMPjKzT81sRLntW83seTP7wsweiPVIql3h3mJ+OWkuPdo04Yaze0cRgohIJMJMEPGmU/MK6w2AbOBbwBjgMTNrGdt+GnAjcCLQA/jxAR9gNtbMZprZzIKCguRFXs59ry5i7bbdPHDpAJWWRKReCTNB5ANdyq13BtbGafOSu+9z9xXAYoKEkQ98EStPFQMvAsdX/AB3f8Tdc9w9p23btkk/gY+XbeSpT1fyk6FZnNCtddKPLyJSk4WZIGYA2WaWZWbpwGhgcoU2LwLDAMysDUFpaXls31ZmVvatPxzIpRrtKirmlknz6J7ZmBvPObo6P1pEpEYILUHE/vK/FpgGLAQmuPsCMxtnZiNjzaYBm8wsF3gHuMndN7l7CUF56S0zm0dQrno0rFjjuf+1xazavIv7Lh5Ao3SVlkSk/jH3ipcFaqecnByfOXNmUo712YrNXPbIJ/zo5O7cObJfUo4pIlITmdnn7p4T7z09SV3B7qISbp44h86tGnHzCJWWRKT+0lhMFfz+9cV8uWkX/7pqCI3T9esRkfpLPYhyPl+5mb99tIIfDOnKKT3bRB2OiEiklCBi9uwr4aaJc+nYohG3ntcn6nBERCKnGgpw2cOfsGrzLtZt28NTPx1M04b6tYiIqAdBMJzGum17GH1iF07LTv4DdyIitVG9TxB7i0tYXrCT9NQUbjtfpSURkTL1PkFs2L4XgKw2jWmekRZxNCIiNUe9L7Z3ad2YYzs1xyze2IIiIvVXve9BAEoOIiJx1PseBMCzV58cdQgiIjWOehAiIhKXEoSIiMSlBCEiInEpQYiISFxKECIiEpcShIiIxKUEISIicSlBiIhIXEoQIiISl7l71DEkhZkVACuP4BBtgI1JCidKdeU8QOdSU9WVc6kr5wFHdi7d3D3uPAd1JkEcKTOb6e45UcdxpOrKeYDOpaaqK+dSV84DwjsXlZhERCQuJQgREYlLCeIbj0QdQJLUlfMAnUtNVVfOpa6cB4R0LroGISIicakHISIicSlBxJjZ3WY218xmm9nrZtYx6piqysweMLNFsfN5wcxaRh1TVZnZpWa2wMxKzazW3XFiZiPMbLGZ5ZnZLVHHcyTM7HEz22Bm86OO5UiYWRcze8fMFsb+bV0fdUxVZWYZZvaZmc2JnctdST2+SkwBM2vu7ttjy/8O9HX3n0UcVpWY2TnA2+5ebGb3Abj7LyMOq0rMrA9QCjwM3OjuMyMOKWFmlgosAc4G8oEZwBh3z400sCoys9OBQuBJd+8fdTxVZWYdgA7uPsvMmgGfA9+tjf9dLJgvuYm7F5pZGvAhcL27f5qM46sHEVOWHGKaALU2c7r76+5eHFv9FOgcZTxHwt0XuvviqOOoosFAnrsvd/ciYDxwYcQxVZm7vw9sjjqOI+Xu69x9Vmx5B7AQ6BRtVFXjgcLYalrslbTvLiWIcszsHjNbDfwAuD3qeJLkJ8CrUQdRT3UCVpdbz6eWfhHVVWbWHTgOmB5tJFVnZqlmNhvYALzh7kk7l3qVIMzsTTObH+d1IYC7/8rduwBPA9dGG+3BHepcYm1+BRQTnE+Nlci51FIWZ1ut7ZnWNWbWFJgE/EeFCkKt4u4l7j6IoFIw2MySVv5rkKwD1QbuflaCTf8FTAHuCDGcI3KoczGzHwEXAGd6Db/QdBj/XWqbfKBLufXOwNqIYpFyYvX6ScDT7v581PEkg7tvNbN3gRFAUm4kqFc9iIMxs+xyqyOBRVHFcqTMbATwS2Cku++KOp56bAaQbWZZZpYOjAYmRxxTvRe7sPs3YKG7/yHqeI6EmbUtu0vRzBoBZ5HE7y7dxRRjZpOAownumFkJ/Mzd10QbVdWYWR7QENgU2/RpLb4jaxTwP0BbYCsw293PjTaqxJnZecAfgVTgcXe/J+KQqszMngG+RTBy6HrgDnf/W6RBVYGZnQp8AMwj+P8d4DZ3nxpdVFVjZgOAfxD8+0oBJrj7uKQdXwlCRETiUYlJRETiUoIQEZG4lCBERCQuJQgREYlLCUJEROJSghA5DGZWeOhWB91/opn1iC03NbOHzWxZbCTO981siJmlx5br1YOsUvMoQYhUEzPrB6S6+/LYpscIBr/Ldvd+wI+BNrGB/d4CLoskUJEYJQiRKrDAA7Exo+aZ2WWx7Slm9r+xHsErZjbVzC6J7fYD4KVYu57AEODX7l4KEBv1dUqs7Yux9iKRURdWpGouAgYBAwmeLJ5hZu8DQ4HuwLFAO4KhpB+P7TMUeCa23I/gqfCSSo4/HzgxlMhFEqQehEjVnAo8ExtJcz3wHsEX+qnAc+5e6u5fAe+U26cDUJDIwWOJoyg2oY1IJJQgRKom3lDeB9sOsBvIiC0vAAaa2cH+H2wI7KlCbCJJoQQhUjXvA5fFJmtpC5wOfEYw5ePFsWsR7QkGtyuzEOgF4O7LgJnAXbHRRTGz7LI5MMwsEyhw933VdUIiFSlBiFTNC8BcYA7wNnBzrKQ0iWAeiPkE82hPB7bF9pnC/gnj34CjgDwzmwc8yjfzRQwDat3oolK3aDRXkSQzs6axSeQzCXoVQ939q9h4/e/E1iu7OF12jOeBW2vxfNxSB+guJpHkeyU2iUs6cHesZ4G77zazOwjmpV5V2c6xyYVeVHKQqKkHISIicekahIiIxKUEISIicSlBiIhIXEoQIiISlxKEiIjEpQQhIiJx/X/Smsz2nh4aZwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "# train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores =  np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "#train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "#train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    #pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'accuary' )\n",
    "pyplot.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l2\n",
      "0.1\n"
     ]
    }
   ],
   "source": [
    "print(grid.best_params_['penalty'])\n",
    "print(grid.best_params_['C'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.36167135,  0.99773092, -0.0648661 ,  0.05966521, -0.04513553,\n",
       "         0.54600449,  0.25990383,  0.15544357]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_estimator_.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>coeffient</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Plasma_glucose_concentration</td>\n",
       "      <td>[0.9977309169127505]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>BMI</td>\n",
       "      <td>[0.5460044910698012]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pregnants</td>\n",
       "      <td>[0.36167135139108064]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Diabetes_pedigree_function</td>\n",
       "      <td>[0.2599038326077313]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Age</td>\n",
       "      <td>[0.15544357341114548]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Triceps_skin_fold_thickness</td>\n",
       "      <td>[0.059665209790310966]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>serum_insulin</td>\n",
       "      <td>[-0.04513553103782168]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blood_pressure</td>\n",
       "      <td>[-0.06486609661371805]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        columns               coeffient\n",
       "1  Plasma_glucose_concentration    [0.9977309169127505]\n",
       "5                           BMI    [0.5460044910698012]\n",
       "0                     pregnants   [0.36167135139108064]\n",
       "6    Diabetes_pedigree_function    [0.2599038326077313]\n",
       "7                           Age   [0.15544357341114548]\n",
       "3   Triceps_skin_fold_thickness  [0.059665209790310966]\n",
       "4                 serum_insulin  [-0.04513553103782168]\n",
       "2                blood_pressure  [-0.06486609661371805]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"columns\":list(feat_names), \"coeffient\":list(grid.best_estimator_.coef_.T)})\n",
    "df.sort_values(by=['coeffient'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
